#!/bin/python
import os
import sys
import time
import os.path

from datetime import date, timedelta
from datetime import datetime

import threading

def usage():
    print """usage zipraw.py daytime input-file-path ...
    eg: zipraw.py first/last/all 2015-01-01 /usr/java/rawdata/netty/2015-01-01
    """ 
def array_exists(files, dst_file):
    for f in files:
        if f == dst_file:
            return True
    return False

def write_index_file(index_file, newname):
    with open(index_file, "a+") as fp:
        fp.write("%s\n" % newname)
        print "write index %s to file:%s" % (newname, index_file)

def write_data_file(data_file, day, lines):
    with open(data_file, "a+") as fp:
        for k,v in lines.items():
            fp.write("%s|%s|%s\n"%(day,k,";".join((str(i[0])+","+str(i[1])) for i in v)))


class Zip():
    def __init__(self, lines, input_file):
        self.lines = lines
        self.input_file = input_file

    def run(self):
        with open(self.input_file) as fp:

            start = datetime.now()
            #lines = output_lines
            for line in fp:
                #print line
                line_arr = line.split('.')
                if len(line_arr) >= 5:
                    day = line_arr[0]
                    if day != today or len(day) != 10:
                        continue
                    device = line_arr[1]
                    mac = line_arr[2]
                    rssi_time = line_arr[3]
                    rssi = line_arr[4]

                    rssi_time_arr = rssi_time.split(':')
                    if len(rssi_time) != 8 or len(rssi_time_arr) != 3:
                        print "error parse %s rssi time, %s,%s" % (self.input_file, mac,rssi_time)
                        continue

                    time_delta = 3600*(int(rssi_time_arr[0])) + 60*(int(rssi_time_arr[1]))+ int(rssi_time_arr[2]) # rssi_datetime - prev_datetime
                    

                    # new_time = day+ " " + rssi_time
                    #new_datetime = dateutil.parser.parse(new_time)
                    key = device+"|" + mac
                    t = (time_delta, rssi)
                    if self.lines.has_key(key):
                        v = self.lines[key]
                        v.append(t)
                    else:
                        self.lines[key] = [t]

            end = datetime.now()
            delta = end - start
            print "read file:%s time:%d, lines:%d\n" % (self.input_file, delta.seconds, len(self.lines))



if len(sys.argv)<2:
    print "ERROR len(sys.argv)"
    usage()
    exit(-1)

runtype = sys.argv[1]
runday = sys.argv[2]


if runday == "today":
    today = date.today()
    today = today.strftime('%Y-%m-%d')
elif runday == "yesterday":
    today = date.today() - timedelta(1)
    today = today.strftime('%Y-%m-%d')
else:
    today = runday

if len(today) != 10:
    print "ERROR len(today) , [%s]" % today
    usage() 
    exit(-1)



input_path = sys.argv[3]
if not os.path.exists(input_path):
    print "ERROR, input file path:%s not exists!" % input_path



lock_path = os.path.join(input_path, "rawzip."+today+".lock")

output_path_filename = os.path.join(input_path, "rawzip.%s.rz" % today)
index_path_filename = os.path.join(input_path, "rawzip."+today+".idx")
zipfile = os.path.join(input_path, "rawzip.%s.txt" % today)
log_filename = os.path.join(input_path, "rawzip.%s.log" % today)

if runtype == 'all':
    if os.path.exists(lock_path):
        os.remove(lock_path)
    if os.path.exists(index_path_filename):
        os.remove(index_path_filename)
    if os.path.exists(output_path_filename):
        os.remove(output_path_filename)
    if os.path.exists(zipfile):
        os.remove(zipfile)

if os.path.exists(lock_path):
    print 'ERROR, another zip running, exit. Please rm lock file %s' % lock_path
    exit(-1)

logger = open(log_filename, "a+")

with open(lock_path, "w+") as fp:
    fp.write("lock")

# index_fp = open(index_path_filename, "a+")
index_files = []
if os.path.exists(index_path_filename):
    with open(index_path_filename) as index_fp:
        for line in index_fp:
            index_files.append(line[:-1])

print "index files: %s" % index_files
new_raw_files = []
will_raw_files = []
for rawfile in os.listdir(input_path):
    #print "list dir file: %s" % rawfile

    if rawfile.startswith("rawdata.%s" % today) and rawfile.endswith(".txt"):
        splits = rawfile.split('.')
        if len(splits) != 4:
            continue
        idx = int(splits[2])
        new_raw_files.append((idx, rawfile))


#sorted(new_raw_files,key=lambda x:int(x[0]))
new_raw_files.sort(key=lambda x:int(x[0]))
#print new_raw_files
if runtype == 'first':
    #remove last one .txt
    new_raw_files = new_raw_files[:-1]
#print new_raw_files
for file_tuple in new_raw_files:
    #print file_tuple
    rawfile = file_tuple[1]
    if array_exists(index_files, rawfile):
        print("raw file %s has ziped." % rawfile )
    else:
        print "will zip file: %s" % rawfile
        will_raw_files.append(rawfile)



for rawfile in will_raw_files:
    start_time = datetime.now()
    lines = {}
    rawfile_path = os.path.join(input_path, rawfile)
    ziper = Zip(lines, rawfile_path)
    print "start zip raw file: %s" % rawfile_path
    ziper.run()
    
    write_data_file(output_path_filename, today, lines)
    write_index_file(index_path_filename, rawfile)
    end_time = datetime.now()
    delta = end_time-start_time
    logger.write("zip txt file %s times:%d s, lines:%d\n" % (rawfile_path, delta.seconds, len(lines)))

if runtype == 'last' or runtype == 'all':
    new_lines = {}
    print "last run zip tmp file %s" % output_path_filename
    start_time = datetime.now()
    with open(output_path_filename) as fp:
        
        for line in fp:
            line_arr = line[:-1].split('|')
            key = '|'.join(line_arr[:3])
            v = line_arr[3]
            if new_lines.has_key(key):
                new_val = new_lines[key]
                new_val.append(v)
            else:
                new_lines[key] = [v]

    with open(zipfile, "w+") as fp:
        for k,v in new_lines.items():
            fp.write('%s|%s\n' % (k, ";".join(v)))
    end_time = datetime.now()
    delta = end_time-start_time
    logger.write("rezip rz file %s times:%d s, lines:%d\n" % (output_path_filename, delta.seconds, len(new_lines)))



if os.path.exists(lock_path):
    os.remove(lock_path)

# for inputfile in sys.argv[2:]:
#     with open(inputfile) as fp:
        
#         start = datetime.now()
#         lines = {}
#         for line in fp:
#             #print line
#             line_arr = line.split('.')
#             if len(line_arr) >= 5:
#                 day = line_arr[0]
#                 if day != today or len(day) != 10:
#                     continue
#                 device = line_arr[1]
#                 mac = line_arr[2]
#                 rssi_time = line_arr[3]
#                 rssi = line_arr[4]

#                 rssi_time_arr = rssi_time.split(':')
#                 if len(rssi_time) != 8 or len(rssi_time_arr) != 3:
#                     print "error parse rssi time, %s,%s" % (mac,rssi_time)
#                     continue

#                 time_delta = 3600*(int(rssi_time_arr[0])) + 60*(int(rssi_time_arr[1]))+ int(rssi_time_arr[2]) # rssi_datetime - prev_datetime
                

#                 key = device+"|" + mac
#                 t = (time_delta, rssi)
#                 if lines.has_key(key):
#                     v = lines[key]
#                     v.append(t)
#                 else:
#                     lines[key] = [t]
                    
                
#         end = datetime.now()
#         delta = end - start
#         print "read time:%d, lines:%d\n" % (delta.seconds, len(lines))


#         for k,v in lines.items():
#             newraw_file.write("%s|%s|%s\n"%(first_datetime,k, ";".join((str(i[0])+","+str(i[1])) for i in v)))


#         delta = datetime.now() - end
#         lines.clear()

#         print "sort time:%d\n" % (delta.seconds)


#print lines

# for k,v in lines.items():
#     sorted_v = v # sorted(v, key=lambda x:x[0])
#     first_time = sorted_v[0][0]
#     timerssi_tuples = []
    # for tuple_value in sorted_v:
    #     if len(tuple_value) != 2:
    #         continue
    #     rssi_time = tuple_value[0]
    #     rssi_time_arr = rssi_time.split(':')
    #     if len(rssi_time_arr) != 3:
    #         print "error parse rssi time, %s,%s" % (k,rssi_time)
    #         continue

    #     time_delta = 3600*(int(rssi_time_arr[0])) + 60*(int(rssi_time_arr[1]))+ int(rssi_time_arr[2]) # rssi_datetime - prev_datetime
    #     rssi = int(tuple_value[1])
    #     timerssi_tuples.append((time_delta, rssi))
        
    # newraw_file.write("%s|%s|%s\n"%(k,first_datetime, ";".join((str(i[0])+","+str(i[1])) for i in v)))


# newraw_file.close()


# input_argv = sys.argv[2:]
# threads = []
# cnt = 0
# for inputfile in input_argv:
#     output_lines.append({})
#     thread = ZipThread(output_lines[cnt], inputfile)
#     threads.append(thread)
#     cnt += 1

# for thread in threads:
#     thread.start()

# for thread in threads:
#     thread.join()


# print "all thead finished. "

# for lines in output_lines:
#     print "lines size is %d" % (len(lines))





